#!/bin/bash

function task_job() {
jdbcUrl="$1"
db="$2"
mysqlTable="$3"
hiveTable="$4"
column="${5}"
sub="${6}"
threadNumber="${7}"
typed="${8}"
dt="{{ execution_date | cst_ds }}"
nextdt="{{ execution_date | date_add(1) | cst_ds }}"
#dt="2023-01-31"
#nextdt="2023-02-01"
env="prod"
username='bigdata_sync'
password='eAvxawdHuzofupUD'

json='{
"reader":{
"connect":{
"url":"'"${jdbcUrl}"'",
"username":"'"${username}"'",
"password":"'"${password}"'",
"driver":"com.mysql.cj.jdbc.Driver"
},
"dbtype":"mysql",
"tableName":"'"${db}.${mysqlTable}"'",
"subTableList":"'"${sub}"'",
"where":"'"${column} between str_to_date('${dt} 00:00:00', '%Y-%m-%d %H:%i:%s') and str_to_date('${nextdt} 00:00:00', '%Y-%m-%d %H:%i:%s')"'",
"query":"'""'",
"splitColumn":"'"${column}"'",
"equalitySectioning":0,
"containsnull":0,
"fetchsize":"1024",
"threadNumber":5
},
"channel":{
"filterAbnormalCharacter":0
},
"writer":{
"dag":"spmi_ods",
"task_id":"spmi_ods__spmi_operation_bill",
"dbtype":"hive",
"tableName":"'"${hiveTable}"'",
"database":"spmi_ods",
"writeMode": "'"${typed}"'",
"partitionColumn":"dt",
"partitionValue":"'"${dt}"'"},
"settting":{
"env":"'"${env}"'"}
}'

/usr/hdp/3.1.4.0-315/spark2/bin/spark-submit \
--master yarn \
--deploy-mode cluster \
--driver-memory 18G \
--driver-cores 4 \
--executor-memory 8G \
--executor-cores 2 --queue pro  \
--num-executors ${threadNumber}  \
--conf spark.driver.maxResultSize=6g  \
--class com.yunlu.bigdata.jobs.synchrotool.DataSynchDriver \
hdfs:///scheduler/jms/spark/sync/mysql/spark_sync.jar  "$json"
echo ${json} >> /jms_ods/spmi_operation_bill.log
echo "任务执行成功: ${dt} ${hiveTable} ${db} ${sub}"
}


jdbcUrl1='{{var.json.mysql_spmi.db07}}'
jdbcUrl2='{{var.json.mysql_spmi.db07_1}}'
jdbcUrl3='{{var.json.mysql_spmi.db3_opt}}'
jdbcUrl4='{{var.json.mysql_spmi.db3_opt_01}}'
jdbcUrl5='{{var.json.mysql_spmi.db4_opt}}'
jdbcUrl6='{{var.json.mysql_spmi.db4_opt_01}}'


mTab="spmi_operation_bill"
hTab="spmi_operation_bill"
col='update_time'

# 2023-08-16 off line by kumiler
#task_job  ${jdbcUrl1}  yl_spmibill_7_1    ${mTab} ${hTab} ${col} "[1,2];[0,7]"  8  overwrite
#task_job  ${jdbcUrl1}  yl_spmibill_7_1    ${mTab} ${hTab} ${col} "[3,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_2    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_3    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_4    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_5    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_6    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_7    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_8    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_9    ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_10   ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_11   ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl1}  yl_spmibill_7_12   ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#wait
#task_job  ${jdbcUrl2}  yl_spmibill_117_1  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_2  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_3  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_4  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_5  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_6  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_7  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_8  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_9  ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_10 ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_11 ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#task_job  ${jdbcUrl2}  yl_spmibill_117_12 ${mTab} ${hTab} ${col} "[1,31];[0,7]" 3  append    &
#wait

#2022-01-20号新加库表实例
task_job  ${jdbcUrl1}  yl_spmibill_opt_7_1      ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
task_job  ${jdbcUrl1}  yl_spmibill_opt_117_1    ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
task_job  ${jdbcUrl1}  yl_spmibill_opt_127      ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
wait
task_job  ${jdbcUrl2}  yl_spmibill_opt_7_2      ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
task_job  ${jdbcUrl2}  yl_spmibill_opt_117_2    ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
wait
task_job  ${jdbcUrl3}  yl_spmibill_opt_7_3      ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
wait
task_job  ${jdbcUrl4}  yl_spmibill_opt_117_3    ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
wait
task_job  ${jdbcUrl5}  yl_spmibill_opt_7_4      ${mTab} ${hTab} ${col} "[0,255]" 3  append    &
wait
task_job  ${jdbcUrl6}  yl_spmibill_opt_117_4    ${mTab} ${hTab} ${col} "[0,255]" 3  append    &

wait
echo "${hTab} 表所有任务执行成功! "
